pisa

library(dplyr)
library(intsvy)
library(haven)
library(caret)
library(ggplot2)
library(tidyr)
library(glmnet)
library(caTools)
library(readxl)
library(tidyverse)
library(writexl)
library(caTools)
library(xgboost)
library(DiagrammeR)

temp_metrics <- read_xlsx(path = "/Users/mauriciodarcourt/Downloads/bkg/pisa_ms_bkg_read_stu_compendium.xlsx")

New names:

col.readvars <- temp_metrics$metrics

stu_headers <- read_xlsx(path = '/Users/mauriciodarcourt/Documents/Pisa/data/2018/headers.xlsx')
stu_headers_list <- stu_headers$stu_headers

stu_headers

######Grabbing all variables measured in pisa

df <- data.frame(stu_headers)
write_xlsx(df,path = "/Users/mauriciodarcourt/Documents/Pisa/data/2018/headers.xlsx")

check <- function(list1,list2){
  
  temp_list <- list()
  for(x in list1){
    if ((x %in% (list2))){
      temp_list <- append(temp_list,x)
    }
  }
return(temp_list)
}

a.check <- check(col.readvars,stu_headers_list)
a <- do.call(rbind.data.frame,a.check)

pisa <- pisa.select.merge(folder = "/Users/mauriciodarcourt/Documents/Pisa/data/2018/final",student.file = "CY07_MSU_STU_QQQ.sav", school.file = "CY07_MSU_SCH_QQQ.sav",student= a.check

)

###CLeaning dataset and piecing together

#identifier data[,c("CNT",)]
id <- data.frame(pisa$CNT)
body <- pisa[,names(pisa) %in% a$c..ST004D01T....ST011Q01TA....ST011Q02TA....ST011Q03TA....ST011Q04TA...]

start = match("PV1READ",names(pisa))
end = match("PV10READ",names(pisa))

score<- data.frame(pisa[,start:end])
score <- rowMeans(score)
clean.data <- data.frame(id , body, score)
clean.data <- clean.data[complete.cases(clean.data),]
cc <- sum(complete.cases(clean.data))

table(clean.data$pisa.CNT)

Factors for categorical variables

for (x in 1:90){
  
  clean.data[,x:x] <- as.factor(clean.data[,x:x])  
}
  

Create training and testing split

set.seed(101) 
sample = sample.split(clean.data$REPEAT, SplitRatio = .7)
train = subset(clean.data, sample == TRUE)
test  = subset(clean.data, sample == FALSE)

linear model

model <- lm(score ~ . ,data = train,)

lm model results

summary(model)

Call:
lm(formula = score ~ ., data = train)

Residuals:
    Min      1Q  Median      3Q     Max 
-302.00  -40.44    1.04   41.70  361.26 

Coefficients:
             Estimate Std. Error t value Pr(>|t|)    
(Intercept) 256.02030    6.00844  42.610  < 2e-16 ***
pisa.CNTARG  17.32432    1.92989   8.977  < 2e-16 ***
pisa.CNTAUS  56.26976    1.79885  31.281  < 2e-16 ***
pisa.CNTAUT  43.91700    2.00704  21.882  < 2e-16 ***
pisa.CNTBGR  23.91467    2.20471  10.847  < 2e-16 ***
pisa.CNTBIH  -9.04091    2.00089  -4.518 6.23e-06 ***
pisa.CNTBLR  40.12377    1.96796  20.388  < 2e-16 ***
pisa.CNTBRA  34.01561    1.99909  17.016  < 2e-16 ***
pisa.CNTBRN  17.81473    2.07986   8.565  < 2e-16 ***
pisa.CNTCHE  45.64556    2.19720  20.774  < 2e-16 ***
pisa.CNTCHL  37.19875    1.98780  18.714  < 2e-16 ***
pisa.CNTCOL  37.62220    1.99549  18.854  < 2e-16 ***
pisa.CNTCRI  26.75729    1.99062  13.442  < 2e-16 ***
pisa.CNTDEU  71.36953    2.49229  28.636  < 2e-16 ***
pisa.CNTDOM  -0.18863    3.52573  -0.054 0.957333    
pisa.CNTESP  35.45645    1.62633  21.801  < 2e-16 ***
pisa.CNTEST  64.21085    1.94244  33.057  < 2e-16 ***
pisa.CNTFRA  59.03773    2.07178  28.496  < 2e-16 ***
pisa.CNTGBR  57.58066    2.50589  22.978  < 2e-16 ***
pisa.CNTGRC  29.65277    1.93754  15.304  < 2e-16 ***
pisa.CNTHKG  96.49169    1.94532  49.602  < 2e-16 ***
pisa.CNTHRV  42.62936    1.95849  21.766  < 2e-16 ***
pisa.CNTHUN  30.91105    2.00529  15.415  < 2e-16 ***
pisa.CNTIDN  19.02264    1.81406  10.486  < 2e-16 ***
pisa.CNTIRL  62.52588    2.05116  30.483  < 2e-16 ***
pisa.CNTISL  28.00336    2.43235  11.513  < 2e-16 ***
pisa.CNTITA  45.64269    1.78186  25.615  < 2e-16 ***
pisa.CNTJOR  46.63602    1.93358  24.119  < 2e-16 ***
pisa.CNTKAZ   9.94730    1.68079   5.918 3.26e-09 ***
pisa.CNTKOR  66.19220    1.84548  35.867  < 2e-16 ***
pisa.CNTKSV -42.03000    2.18339 -19.250  < 2e-16 ***
pisa.CNTLTU  27.81497    1.90142  14.629  < 2e-16 ***
pisa.CNTLVA  31.19928    1.97859  15.768  < 2e-16 ***
pisa.CNTMAC  94.41982    2.02775  46.564  < 2e-16 ***
pisa.CNTMAR  16.53377    2.99337   5.523 3.33e-08 ***
pisa.CNTMDA  19.29362    2.10442   9.168  < 2e-16 ***
pisa.CNTMEX  27.42295    2.11295  12.979  < 2e-16 ***
pisa.CNTMLT  22.72962    2.28274   9.957  < 2e-16 ***
pisa.CNTMNE   5.97117    1.99431   2.994 0.002753 ** 
pisa.CNTPAN  15.92967    2.99796   5.314 1.08e-07 ***
pisa.CNTPER  22.11123    2.37443   9.312  < 2e-16 ***
pisa.CNTPHL -17.57170    2.02450  -8.680  < 2e-16 ***
pisa.CNTPOL  55.90513    1.94072  28.806  < 2e-16 ***
pisa.CNTPRT  62.45301    1.97146  31.679  < 2e-16 ***
pisa.CNTQAZ  -0.31301    2.32582  -0.135 0.892943    
pisa.CNTQMR  55.70492    2.68312  20.761  < 2e-16 ***
pisa.CNTQRT  44.20914    1.98467  22.275  < 2e-16 ***
pisa.CNTROU   5.63293    2.16801   2.598 0.009372 ** 
pisa.CNTRUS  54.41711    1.89703  28.685  < 2e-16 ***
pisa.CNTSAU  24.61949    2.17391  11.325  < 2e-16 ***
pisa.CNTSRB  21.10570    2.01440  10.477  < 2e-16 ***
pisa.CNTSVK  36.74493    1.98036  18.555  < 2e-16 ***
pisa.CNTSVN  42.02605    1.96761  21.359  < 2e-16 ***
pisa.CNTTAP  61.37158    1.81664  33.783  < 2e-16 ***
pisa.CNTTHA  13.94142    1.76607   7.894 2.95e-15 ***
pisa.CNTTUR  37.61933    1.87832  20.028  < 2e-16 ***
pisa.CNTUKR  36.16946    1.98746  18.199  < 2e-16 ***
pisa.CNTURY  41.32574    2.50873  16.473  < 2e-16 ***
ST004D01T2   -3.11916    0.37025  -8.425  < 2e-16 ***
ST011Q01TA2  -2.76037    0.66624  -4.143 3.43e-05 ***
ST011Q02TA2   3.48202    0.47768   7.289 3.13e-13 ***
ST011Q03TA2  -0.41959    0.57694  -0.727 0.467062    
ST011Q04TA2  -7.98281    0.71205 -11.211  < 2e-16 ***
ST011Q05TA2   2.37556    0.37898   6.268 3.66e-10 ***
ST011Q06TA2 -11.34632    0.85755 -13.231  < 2e-16 ***
ST011Q07TA2  -5.11854    0.43327 -11.814  < 2e-16 ***
ST011Q08TA2   2.59712    0.41885   6.201 5.64e-10 ***
ST011Q09TA2   0.27528    0.38448   0.716 0.473994    
ST011Q10TA2  -0.32216    0.50165  -0.642 0.520749    
ST011Q11TA2  -1.48280    0.39362  -3.767 0.000165 ***
ST011Q12TA2  -7.26796    0.66187 -10.981  < 2e-16 ***
ST011Q16NA2   4.65719    0.39166  11.891  < 2e-16 ***
ST012Q01TA2  -0.93472    1.56968  -0.595 0.551523    
ST012Q01TA3  -5.13071    1.59497  -3.217 0.001297 ** 
ST012Q01TA4  -7.64284    1.63937  -4.662 3.13e-06 ***
ST012Q02TA2   3.19360    0.56556   5.647 1.64e-08 ***
ST012Q02TA3   2.79753    0.69284   4.038 5.40e-05 ***
ST012Q02TA4  -4.33444    0.90187  -4.806 1.54e-06 ***
ST012Q03TA2   5.04776    0.72737   6.940 3.95e-12 ***
ST012Q03TA3   8.77723    0.83333  10.533  < 2e-16 ***
ST012Q03TA4   9.64191    1.03382   9.327  < 2e-16 ***
ST012Q05NA2   9.28935    1.21875   7.622 2.51e-14 ***
ST012Q05NA3  24.71230    1.22774  20.128  < 2e-16 ***
ST012Q05NA4  28.09491    1.17440  23.923  < 2e-16 ***
ST012Q06NA2   7.75186    0.87710   8.838  < 2e-16 ***
ST012Q06NA3  15.05370    0.98904  15.221  < 2e-16 ***
ST012Q06NA4  23.60916    1.11561  21.163  < 2e-16 ***
ST012Q07NA2  -2.08643    0.44731  -4.664 3.10e-06 ***
ST012Q07NA3  -4.25596    0.60734  -7.008 2.44e-12 ***
ST012Q07NA4  -8.61381    0.79893 -10.782  < 2e-16 ***
ST012Q08NA2  -5.56082    0.48702 -11.418  < 2e-16 ***
ST012Q08NA3  -8.36379    0.94388  -8.861  < 2e-16 ***
ST012Q08NA4 -14.33083    1.21721 -11.774  < 2e-16 ***
ST012Q09NA2   1.62448    0.42272   3.843 0.000122 ***
ST012Q09NA3   5.90410    0.53559  11.024  < 2e-16 ***
ST012Q09NA4   7.66140    0.56999  13.441  < 2e-16 ***
ST013Q01TA2   4.05861    0.56189   7.223 5.11e-13 ***
ST013Q01TA3  15.53142    0.56451  27.513  < 2e-16 ***
ST013Q01TA4  22.85638    0.68149  33.539  < 2e-16 ***
ST013Q01TA5  31.57393    0.77486  40.748  < 2e-16 ***
ST013Q01TA6  26.27137    0.93626  28.060  < 2e-16 ***
ST097Q03TA2   5.00574    0.81668   6.129 8.84e-10 ***
ST097Q03TA3   8.30275    0.79240  10.478  < 2e-16 ***
ST097Q03TA4  10.51472    0.83246  12.631  < 2e-16 ***
ST097Q04TA2   5.06513    1.02245   4.954 7.28e-07 ***
ST097Q04TA3   7.11848    0.99267   7.171 7.48e-13 ***
ST097Q04TA4   7.35922    1.02541   7.177 7.17e-13 ***
ST097Q05TA2   2.35245    0.88831   2.648 0.008092 ** 
ST097Q05TA3   2.66555    0.86902   3.067 0.002160 ** 
ST097Q05TA4   1.05630    0.90465   1.168 0.242956    
ST100Q01TA2   7.50156    0.47649  15.743  < 2e-16 ***
ST100Q01TA3   3.05291    0.61135   4.994 5.93e-07 ***
ST100Q01TA4   0.48626    0.85474   0.569 0.569429    
ST100Q02TA2   0.08236    0.50110   0.164 0.869449    
ST100Q02TA3  -4.58736    0.65048  -7.052 1.77e-12 ***
ST100Q02TA4  -5.95949    1.00235  -5.946 2.76e-09 ***
ST100Q03TA2  -0.54689    0.52674  -1.038 0.299151    
ST100Q03TA3  -2.19495    0.70961  -3.093 0.001981 ** 
ST100Q03TA4  -0.71820    1.08372  -0.663 0.507514    
ST100Q04TA2   3.64700    0.47835   7.624 2.47e-14 ***
ST100Q04TA3  -0.28561    0.61361  -0.465 0.641607    
ST100Q04TA4  -0.97535    0.92753  -1.052 0.293009    
ST102Q01TA2   3.08465    0.44921   6.867 6.59e-12 ***
ST102Q01TA3   2.15937    0.59323   3.640 0.000273 ***
ST102Q01TA4   7.78529    0.91507   8.508  < 2e-16 ***
ST102Q02TA2  -1.05651    0.45019  -2.347 0.018936 *  
ST102Q02TA3  -3.79223    0.59729  -6.349 2.17e-10 ***
ST102Q02TA4  -4.23067    1.01932  -4.151 3.32e-05 ***
ST102Q03TA2   8.15359    0.50596  16.115  < 2e-16 ***
ST102Q03TA3  17.66419    0.52444  33.682  < 2e-16 ***
ST102Q03TA4  22.75298    0.66942  33.989  < 2e-16 ***
ST102Q04TA2   4.04863    0.44850   9.027  < 2e-16 ***
ST102Q04TA3   0.98285    0.58213   1.688 0.091342 .  
ST102Q04TA4  -5.32053    0.96577  -5.509 3.61e-08 ***
ST211Q01HA2   5.73926    0.87222   6.580 4.72e-11 ***
ST211Q01HA3   4.22230    0.86934   4.857 1.19e-06 ***
ST211Q01HA4   4.50604    0.97660   4.614 3.95e-06 ***
ST211Q02HA2   0.47429    0.84550   0.561 0.574826    
ST211Q02HA3  -0.73513    0.86714  -0.848 0.396569    
ST211Q02HA4  -0.23275    0.99977  -0.233 0.815913    
ST211Q03HA2   2.28252    0.86330   2.644 0.008196 ** 
ST211Q03HA3   0.92157    0.87594   1.052 0.292760    
ST211Q03HA4   0.43739    0.99429   0.440 0.660009    
ST212Q01HA2   1.73885    0.67172   2.589 0.009636 ** 
ST212Q01HA3   8.73768    0.71936  12.147  < 2e-16 ***
ST212Q01HA4  13.82826    0.79812  17.326  < 2e-16 ***
ST212Q02HA2  -2.28504    0.61730  -3.702 0.000214 ***
ST212Q02HA3  -2.80354    0.67565  -4.149 3.34e-05 ***
ST212Q02HA4  -0.85855    0.75810  -1.133 0.257420    
ST212Q03HA2  -1.08119    0.49989  -2.163 0.030555 *  
ST212Q03HA3  -1.23304    0.56069  -2.199 0.027868 *  
ST212Q03HA4  -2.86103    0.68890  -4.153 3.28e-05 ***
ST104Q02NA2  -3.10876    0.48978  -6.347 2.20e-10 ***
ST104Q02NA3  -3.97609    0.60772  -6.543 6.07e-11 ***
ST104Q02NA4  -6.63978    0.83628  -7.940 2.04e-15 ***
ST104Q03NA2  -3.13579    0.57606  -5.443 5.23e-08 ***
ST104Q03NA3  -3.54989    0.68431  -5.188 2.13e-07 ***
ST104Q03NA4  -3.93112    0.90244  -4.356 1.33e-05 ***
ST104Q04NA2  -1.71789    0.60078  -2.859 0.004245 ** 
ST104Q04NA3  -3.11723    0.69626  -4.477 7.57e-06 ***
ST104Q04NA4  -8.93511    0.86848 -10.288  < 2e-16 ***
ST213Q01HA2   2.40095    1.08925   2.204 0.027510 *  
ST213Q01HA3  -0.64565    1.46814  -0.440 0.660101    
ST213Q01HA4   0.49916    2.18417   0.229 0.819230    
ST213Q02HA2  -5.90268    0.85603  -6.895 5.39e-12 ***
ST213Q02HA3 -13.74261    1.31904 -10.419  < 2e-16 ***
ST213Q02HA4 -15.12378    1.96894  -7.681 1.59e-14 ***
ST213Q03HA2   3.50131    1.27258   2.751 0.005936 ** 
ST213Q03HA3   5.18962    1.52969   3.393 0.000693 ***
ST213Q03HA4  10.45455    2.12121   4.929 8.29e-07 ***
ST213Q04HA2   2.75982    1.13126   2.440 0.014705 *  
ST213Q04HA3   3.37127    1.54724   2.179 0.029341 *  
ST213Q04HA4   1.27455    2.29364   0.556 0.578423    
ST150Q01IA2   0.46951    0.49379   0.951 0.341697    
ST150Q01IA3  -3.54592    0.58700  -6.041 1.54e-09 ***
ST150Q01IA4  -1.84162    0.62443  -2.949 0.003186 ** 
ST150Q02IA2   3.71122    0.42027   8.831  < 2e-16 ***
ST150Q02IA3   2.95807    0.50716   5.833 5.47e-09 ***
ST150Q02IA4   1.76554    0.61477   2.872 0.004081 ** 
ST150Q03IA2  -2.93108    0.48145  -6.088 1.15e-09 ***
ST150Q03IA3  -6.61227    0.58429 -11.317  < 2e-16 ***
ST150Q03IA4  -5.62281    0.63289  -8.884  < 2e-16 ***
ST150Q04HA2  -2.62514    0.49684  -5.284 1.27e-07 ***
ST150Q04HA3  -5.35825    0.55328  -9.685  < 2e-16 ***
ST150Q04HA4   0.50787    0.52528   0.967 0.333618    
ST152Q05IA2   4.98595    0.72922   6.837 8.10e-12 ***
ST152Q05IA3  15.06740    0.77181  19.522  < 2e-16 ***
ST152Q05IA4  20.00016    0.85383  23.424  < 2e-16 ***
ST152Q06IA2  -5.92894    0.53714 -11.038  < 2e-16 ***
ST152Q06IA3  -9.17715    0.59852 -15.333  < 2e-16 ***
ST152Q06IA4  -8.96573    0.74869 -11.975  < 2e-16 ***
ST152Q08IA2  -0.56706    0.70715  -0.802 0.422610    
ST152Q08IA3   2.58506    0.75266   3.435 0.000594 ***
ST152Q08IA4   3.67696    0.83335   4.412 1.02e-05 ***
ST154Q01HA2   7.15002    0.77897   9.179  < 2e-16 ***
ST154Q01HA3   7.81031    0.86292   9.051  < 2e-16 ***
ST154Q01HA4  10.75354    0.86860  12.380  < 2e-16 ***
ST154Q01HA5  28.16206    0.80998  34.769  < 2e-16 ***
ST154Q01HA6  20.33883    1.05259  19.323  < 2e-16 ***
ST160Q01IA2  -1.12688    0.52260  -2.156 0.031061 *  
 [ reached getOption("max.print") -- omitted 115 rows ]
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 61.54 on 138645 degrees of freedom
Multiple R-squared:  0.5663,    Adjusted R-squared:  0.5653 
F-statistic: 576.4 on 314 and 138645 DF,  p-value: < 2.2e-16

accuracy linear model

pred.train <- predict(model, newdata=train) #R2 train
pred.test <- predict(model, newdata=test)   #R2 test

lin.reg.R2.train <- R2(pred.train,train$score)
lin.reg.MAE.train <-mean(abs(train$score-pred.train))
lin.reg.RMSE.train <-sqrt(mean((train$score-pred.train)^2))

lin.reg.R2.test <- R2(pred.test,test$score)                 
lin.reg.MAE.test <-mean(abs(test$score-pred.test))
lin.reg.RMSE.test <-sqrt(mean((test$score-pred.test)^2))

linear.summary <- data.frame(
             train.r2 = lin.reg.R2.train,
             train.MAE = lin.reg.MAE.train,
             train.RMSE = lin.reg.RMSE.train,
             test.r2 = lin.reg.R2.test,
             test.MAE = lin.reg.MAE.test,
             test.RMSE = lin.reg.RMSE.test )

print(linear.summary)

NA

library(rpart)
library(randomForest)

randomForest 4.7-1.1
Type rfNews() to see new features/changes/bug fixes.

Attaching package: ‘randomForest’

The following object is masked from ‘package:ggplot2’:

    margin

The following object is masked from ‘package:dplyr’:

    combine

library(rpart.plot)

cv.trees = train(y = train$score, x = subset(train, select = -c(score))  , method = "rpart", 
                 trControl = trainControl(method = "cv", number = 10), 
                 tuneGrid = data.frame(.cp = seq(.00002,.002,.00002)))
pdf('CART.pdf',12,12)
prp(cv.trees$finalModel,varlen=0,faclen=0,digits=3) 
dev.off()

best.tree <- cv.trees$finalModel

set.seed(123)
best.rf <- randomForest(score~. ,data = train )

#predict cart
cart.train.pred <- predict(best.tree, newdata =train )
cart.test.pred <- predict(best.tree, newdata = test)


CART.R2.train <- R2(cart.train.pred,train$score)
CART.MAE.train <- mean(abs(train$score-cart.train.pred))
CART.RMSE.train <- sqrt(mean((train$score-cart.train.pred)^2))

CART.R2.test <- R2(cart.test.pred,test$score)
CART.MAE.test <- mean(abs(test$score-cart.test.pred))
CART.RMSE.test <- sqrt(mean((test$score-cart.test.pred)^2))

#predict RF

#rf.train.pred <- predict(best.rf,newdata = train)
#rf.test.pred <- predict(best.rf,newdata = test)

#RF.R2.train <- R2(rf.train.pred,train$score)
#RF.MAE.train <- mean(abs(train$score-rf.train.pred))
#RF.RMSE.train <- sqrt(mean((train$score-rf.train.pred)^2))


#RF.R2.test <- R2(rf.test.pred,test$score)
#RF.MAE.test <- mean(abs(test$score-rf.test.pred))
#RF.RMSE.test <- sqrt(mean((test$score-rf.test.pred)^2))

# Summary

a = "Linear"
b = "Cart"
c = "Rf"
summary_statistics <- data.frame(
  row.name = c(a,b,c),
  IS.R2 = c(lin.reg.R2.train,CART.R2.train,RF.R2.train),
  IS.MAE = c(lin.reg.MAE.train,CART.MAE.train,RF.MAE.train),
  IS.RMSE = c(lin.reg.RMSE.train,CART.RMSE.train,RF.RMSE.train),
  OOS.R2 = c(lin.reg.R2.test,CART.R2.test,RF.R2.test),
  OOS.MAE = c(lin.reg.MAE.test,CART.MAE.test,RF.MAE.test),
  OOS.RMSE = c(lin.reg.RMSE.test,CART.RMSE.test,RF.RMSE.test)
)

Error in data.frame(row.name = c(a, b, c), IS.R2 = c(lin.reg.R2.train,  : 
  object 'CART.R2.train' not found

print(summary_statistics)

---

Sub-setting per Country

espana.pisa <- subset(clean.data,clean.data$pisa.CNT == 'ESP')

set.seed(101) 
sample = sample.split(espana.pisa$IMMIG, SplitRatio = .7)
espana.train = subset(espana.pisa, sample == TRUE)
espana.test  = subset(espana.pisa, sample == FALSE)

espana.model <- lm(score ~ . ,data = espana.train[,2:95],)

summary(espana.model)

Call:
lm(formula = score ~ ., data = espana.train[, 2:95])

Residuals:
     Min       1Q   Median       3Q      Max 
-277.043  -37.434    1.903   39.723  196.789 

Coefficients:
              Estimate Std. Error t value Pr(>|t|)    
(Intercept) 322.882263  23.579254  13.693  < 2e-16 ***
ST004D01T2    3.392070   1.266682   2.678 0.007419 ** 
ST011Q01TA2   5.135600   4.019767   1.278 0.201422    
ST011Q02TA2   0.515953   1.865331   0.277 0.782091    
ST011Q03TA2   2.394865   2.471322   0.969 0.332535    
ST011Q04TA2  -9.988764   3.026160  -3.301 0.000967 ***
ST011Q05TA2   5.151523   1.202456   4.284 1.85e-05 ***
ST011Q06TA2   3.647988   4.652173   0.784 0.432971    
ST011Q07TA2  -0.506451   1.422102  -0.356 0.721751    
ST011Q08TA2   0.315544   1.384233   0.228 0.819685    
ST011Q09TA2  -7.783482   1.289482  -6.036 1.63e-09 ***
ST011Q10TA2   3.983660   1.498913   2.658 0.007879 ** 
ST011Q11TA2   0.167433   1.282070   0.131 0.896098    
ST011Q12TA2  -1.324118   4.175492  -0.317 0.751161    
ST011Q16NA2   1.261872   1.320406   0.956 0.339260    
ST012Q01TA2 -14.409598  11.520255  -1.251 0.211032    
ST012Q01TA3 -16.308220  11.489553  -1.419 0.155812    
ST012Q01TA4 -19.801988  11.539004  -1.716 0.086174 .  
ST012Q02TA2   2.696232   2.974606   0.906 0.364735    
ST012Q02TA3   0.372395   3.133594   0.119 0.905405    
ST012Q02TA4  -5.748971   3.692242  -1.557 0.119490    
ST012Q03TA2  20.206356   5.072267   3.984 6.83e-05 ***
ST012Q03TA3  23.874705   5.066896   4.712 2.48e-06 ***
ST012Q03TA4  27.386201   5.389728   5.081 3.81e-07 ***
ST012Q05NA2   5.039166  13.391850   0.376 0.706712    
ST012Q05NA3  28.577543  12.370544   2.310 0.020900 *  
ST012Q05NA4  26.694595  12.249674   2.179 0.029337 *  
ST012Q06NA2   0.487870   4.106162   0.119 0.905425    
ST012Q06NA3   4.037343   4.275209   0.944 0.345006    
ST012Q06NA4  11.082550   4.587517   2.416 0.015717 *  
ST012Q07NA2  -3.778793   2.088585  -1.809 0.070438 .  
ST012Q07NA3  -2.689362   2.519616  -1.067 0.285828    
ST012Q07NA4  -3.644779   3.222655  -1.131 0.258086    
ST012Q08NA2   3.986291   1.329598   2.998 0.002723 ** 
ST012Q08NA3   6.292108   2.246293   2.801 0.005102 ** 
ST012Q08NA4  12.475316   3.490779   3.574 0.000353 ***
ST012Q09NA2  -0.964417   1.505859  -0.640 0.521898    
ST012Q09NA3  -0.457723   1.653966  -0.277 0.781983    
ST012Q09NA4   0.069581   1.751772   0.040 0.968317    
ST013Q01TA2   8.417518   2.646740   3.180 0.001475 ** 
ST013Q01TA3  12.888162   2.515481   5.124 3.05e-07 ***
ST013Q01TA4  18.215682   2.769471   6.577 5.01e-11 ***
ST013Q01TA5  26.059027   2.971004   8.771  < 2e-16 ***
ST013Q01TA6  25.854400   3.367789   7.677 1.77e-14 ***
ST097Q03TA2   2.311513   2.239502   1.032 0.302022    
ST097Q03TA3   0.613683   2.334269   0.263 0.792632    
ST097Q03TA4   0.219764   2.611809   0.084 0.932945    
ST097Q04TA2   6.392354   3.061969   2.088 0.036851 *  
ST097Q04TA3   8.102688   3.040962   2.665 0.007721 ** 
ST097Q04TA4  13.649849   3.194291   4.273 1.94e-05 ***
ST097Q05TA2   1.161526   2.372073   0.490 0.624379    
ST097Q05TA3   0.734002   2.432327   0.302 0.762834    
ST097Q05TA4  -1.348834   2.645092  -0.510 0.610105    
ST100Q01TA2   2.760433   1.670605   1.652 0.098491 .  
ST100Q01TA3   1.494711   2.255465   0.663 0.507533    
ST100Q01TA4   9.742504   3.420963   2.848 0.004409 ** 
ST100Q02TA2   1.324818   1.729493   0.766 0.443684    
ST100Q02TA3  -1.336149   2.154892  -0.620 0.535235    
ST100Q02TA4  -4.986565   3.046303  -1.637 0.101675    
ST100Q03TA2   1.141000   1.861875   0.613 0.540006    
ST100Q03TA3   0.030652   2.629807   0.012 0.990700    
ST100Q03TA4   2.272749   4.294414   0.529 0.596654    
ST100Q04TA2  -2.144699   1.618108  -1.325 0.185054    
ST100Q04TA3  -3.108465   2.065826  -1.505 0.132428    
ST100Q04TA4  -3.562522   3.105880  -1.147 0.251396    
ST102Q01TA2   4.269619   1.497566   2.851 0.004366 ** 
ST102Q01TA3   3.389141   1.946980   1.741 0.081761 .  
ST102Q01TA4   8.657205   2.766024   3.130 0.001754 ** 
ST102Q02TA2  -0.012758   1.478664  -0.009 0.993116    
ST102Q02TA3  -0.962012   1.755564  -0.548 0.583717    
ST102Q02TA4   1.826365   2.569295   0.711 0.477197    
ST102Q03TA2   7.413589   2.144435   3.457 0.000548 ***
ST102Q03TA3  12.393424   2.003683   6.185 6.42e-10 ***
ST102Q03TA4  13.853067   2.153352   6.433 1.30e-10 ***
ST102Q04TA2   5.484402   1.465520   3.742 0.000183 ***
ST102Q04TA3  -0.278748   1.663955  -0.168 0.866963    
ST102Q04TA4  -5.155956   2.476393  -2.082 0.037362 *  
ST211Q01HA2   5.564242   2.828421   1.967 0.049178 *  
ST211Q01HA3  11.382180   2.935283   3.878 0.000106 ***
ST211Q01HA4  16.926616   3.271224   5.174 2.33e-07 ***
ST211Q02HA2  -1.443187   2.621593  -0.550 0.581988    
ST211Q02HA3  -1.585254   2.717147  -0.583 0.559619    
ST211Q02HA4   2.243449   3.180463   0.705 0.480586    
ST211Q03HA2  -1.306564   2.645895  -0.494 0.621452    
ST211Q03HA3  -2.090581   2.821804  -0.741 0.458790    
ST211Q03HA4  -3.329143   3.299000  -1.009 0.312931    
ST212Q01HA2  -0.249902   2.226138  -0.112 0.910621    
ST212Q01HA3   6.095085   2.435317   2.503 0.012336 *  
ST212Q01HA4   9.970125   2.729627   3.653 0.000261 ***
ST212Q02HA2  -0.008257   1.754447  -0.005 0.996245    
ST212Q02HA3  -0.402534   2.019311  -0.199 0.841999    
ST212Q02HA4   0.837744   2.371407   0.353 0.723894    
ST212Q03HA2   0.530426   1.632447   0.325 0.745243    
ST212Q03HA3   0.491869   1.844275   0.267 0.789705    
ST212Q03HA4   1.432523   2.376448   0.603 0.546654    
ST104Q02NA2  -4.208238   1.512532  -2.782 0.005408 ** 
ST104Q02NA3  -6.542152   1.993603  -3.282 0.001036 ** 
ST104Q02NA4  -5.945336   2.932655  -2.027 0.042657 *  
ST104Q03NA2   4.511289   1.714043   2.632 0.008501 ** 
ST104Q03NA3   2.770351   2.105289   1.316 0.188235    
ST104Q03NA4  -0.039022   2.853273  -0.014 0.989089    
ST104Q04NA2  -3.236154   1.719140  -1.882 0.059805 .  
ST104Q04NA3  -6.752144   2.083942  -3.240 0.001199 ** 
ST104Q04NA4 -11.781636   2.780532  -4.237 2.28e-05 ***
ST213Q01HA2  -3.074093   3.566562  -0.862 0.388750    
ST213Q01HA3  -9.615599   4.874298  -1.973 0.048553 *  
ST213Q01HA4 -10.284697   7.277767  -1.413 0.157635    
ST213Q02HA2  -7.335188   2.583584  -2.839 0.004532 ** 
ST213Q02HA3 -15.154136   4.319227  -3.509 0.000452 ***
ST213Q02HA4 -13.769915   6.675602  -2.063 0.039163 *  
ST213Q03HA2  -1.333620   4.105974  -0.325 0.745339    
ST213Q03HA3   0.250221   4.999474   0.050 0.960084    
ST213Q03HA4   1.979411   7.015127   0.282 0.777824    
ST213Q04HA2  -0.192373   3.548297  -0.054 0.956764    
ST213Q04HA3  -4.682299   4.966539  -0.943 0.345820    
ST213Q04HA4 -17.868345   7.426054  -2.406 0.016137 *  
ST150Q01IA2  -0.078713   1.816940  -0.043 0.965446    
ST150Q01IA3  -7.611059   1.985773  -3.833 0.000127 ***
ST150Q01IA4  -9.941790   2.000021  -4.971 6.77e-07 ***
ST150Q02IA2   2.002035   1.571171   1.274 0.202609    
ST150Q02IA3   5.354078   1.689008   3.170 0.001529 ** 
ST150Q02IA4   2.165196   1.902194   1.138 0.255036    
ST150Q03IA2  -1.165111   1.552996  -0.750 0.453130    
ST150Q03IA3  -2.149110   1.826642  -1.177 0.239406    
ST150Q03IA4  -1.018267   1.912107  -0.533 0.594365    
ST150Q04HA2   1.859742   1.715957   1.084 0.278481    
ST150Q04HA3  -0.719656   1.869306  -0.385 0.700255    
ST150Q04HA4   4.810508   1.713291   2.808 0.004998 ** 
ST152Q05IA2   3.873967   1.933300   2.004 0.045115 *  
ST152Q05IA3  10.680883   2.146918   4.975 6.62e-07 ***
ST152Q05IA4   7.131318   2.551218   2.795 0.005195 ** 
ST152Q06IA2  -5.517430   1.410699  -3.911 9.24e-05 ***
ST152Q06IA3 -12.413158   1.818229  -6.827 9.13e-12 ***
ST152Q06IA4 -14.321268   2.681086  -5.342 9.40e-08 ***
ST152Q08IA2   2.257024   1.826134   1.236 0.216501    
ST152Q08IA3   3.314623   2.065974   1.604 0.108658    
ST152Q08IA4   6.117517   2.531116   2.417 0.015668 *  
ST154Q01HA2  -2.296678   2.255966  -1.018 0.308678    
ST154Q01HA3  -2.225732   3.118615  -0.714 0.475432    
ST154Q01HA4   3.129322   2.520947   1.241 0.214511    
ST154Q01HA5  10.361527   2.036336   5.088 3.67e-07 ***
ST154Q01HA6 -15.937030   3.774880  -4.222 2.44e-05 ***
ST160Q01IA2  -2.265870   1.821835  -1.244 0.213626    
ST160Q01IA3  -6.367286   2.295662  -2.774 0.005553 ** 
ST160Q01IA4  -3.042185   2.846365  -1.069 0.285185    
ST175Q01IA2   6.631858   1.733873   3.825 0.000132 ***
ST175Q01IA3   9.602708   2.043982   4.698 2.66e-06 ***
ST175Q01IA4   3.817524   2.445116   1.561 0.118485    
ST175Q01IA5   1.854857   3.101914   0.598 0.549871    
ST161Q01HA2  -0.025814   2.186855  -0.012 0.990582    
ST161Q01HA3  -3.024322   2.195198  -1.378 0.168324    
ST161Q01HA4  -4.948919   2.764939  -1.790 0.073500 .  
ST161Q02HA2   6.359951   3.019807   2.106 0.035220 *  
ST161Q02HA3  17.314054   3.048340   5.680 1.38e-08 ***
ST161Q02HA4  25.719843   3.489595   7.370 1.82e-13 ***
ST161Q03HA2   1.399494   3.870933   0.362 0.717703    
ST161Q03HA3  12.894196   3.695105   3.490 0.000486 ***
ST161Q03HA4  20.156783   3.892201   5.179 2.27e-07 ***
ST161Q06HA2  -9.373312   1.401038  -6.690 2.34e-11 ***
ST161Q06HA3 -17.191579   2.369455  -7.255 4.27e-13 ***
ST161Q06HA4 -29.519778   3.793782  -7.781 7.84e-15 ***
ST161Q07HA2   1.303947   1.825434   0.714 0.475044    
ST161Q07HA3 -10.554082   1.989945  -5.304 1.16e-07 ***
ST161Q07HA4 -11.776440   2.933924  -4.014 6.01e-05 ***
ST161Q08HA2  -0.205893   1.510811  -0.136 0.891603    
ST161Q08HA3 -11.619775   2.089991  -5.560 2.77e-08 ***
ST161Q08HA4  -7.511329   3.697440  -2.031 0.042229 *  
ST184Q01HA2   0.255775   1.539952   0.166 0.868087    
ST184Q01HA3  -4.289143   1.651274  -2.597 0.009404 ** 
ST184Q01HA4  -8.252758   2.205330  -3.742 0.000183 ***
ST185Q01HA2  -3.309946   3.047718  -1.086 0.277485    
ST185Q01HA3 -12.843287   3.097110  -4.147 3.40e-05 ***
ST185Q01HA4 -15.306605   3.363864  -4.550 5.41e-06 ***
ST185Q02HA2   0.322390   3.308922   0.097 0.922386    
ST185Q02HA3  -1.640004   3.426691  -0.479 0.632235    
ST185Q02HA4   0.520311   3.854239   0.135 0.892617    
ST185Q03HA2   1.378203   3.159408   0.436 0.662684    
ST185Q03HA3  -5.911980   3.219652  -1.836 0.066353 .  
ST185Q03HA4 -11.957707   3.509803  -3.407 0.000659 ***
ST208Q01HA2   1.992610   3.418497   0.583 0.559979    
ST208Q01HA3   6.067781   3.522869   1.722 0.085026 .  
ST208Q01HA4   7.117587   3.635804   1.958 0.050298 .  
ST208Q01HA5   4.160375   3.823826   1.088 0.276613    
ST208Q02HA2  -3.567537   2.595514  -1.375 0.169314    
ST208Q02HA3  -7.792128   2.760437  -2.823 0.004769 ** 
ST208Q02HA4 -10.394660   2.948895  -3.525 0.000425 ***
ST208Q02HA5 -11.808154   3.343192  -3.532 0.000414 ***
ST208Q04HA2   4.578753   3.893354   1.176 0.239603    
ST208Q04HA3   7.831928   3.947323   1.984 0.047268 *  
ST208Q04HA4  12.147986   4.003440   3.034 0.002416 ** 
ST208Q04HA5  15.025675   4.163436   3.609 0.000309 ***
ST188Q01HA2   6.271894   4.413694   1.421 0.155343    
ST188Q01HA3  11.940081   4.225658   2.826 0.004728 ** 
ST188Q01HA4  15.822992   4.347201   3.640 0.000274 ***
ST188Q02HA2  18.278342   6.777216   2.697 0.007007 ** 
ST188Q02HA3  13.085821   6.533357   2.003 0.045210 *  
ST188Q02HA4  16.715673   6.591747   2.536 0.011231 *  
ST188Q03HA2  -1.080618   5.242951  -0.206 0.836710    
ST188Q03HA3  -0.222969   5.154052  -0.043 0.965494    
ST188Q03HA4   1.831269   5.280553   0.347 0.728752    
 [ reached getOption("max.print") -- omitted 58 rows ]
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 58.06 on 10954 degrees of freedom
Multiple R-squared:  0.5158,    Adjusted R-squared:  0.5044 
F-statistic:  45.4 on 257 and 10954 DF,  p-value: < 2.2e-16

cv.trees = train(y = espana.train$score, x = subset(espana.train, select = -c(score))  , method = "rpart", 
                 trControl = trainControl(method = "cv", number = 10), 
                 tuneGrid = data.frame(.cp = seq(.00002,.002,.00002)))

Error in train(y = espana.train$score, x = subset(espana.train, select = -c(score)),  : 
  could not find function "train"

set.seed(123)
best.rf <- randomForest(score~. ,data = espana.train[,2:95] )

spain.boost <- 
  
  xgboost(
    data = data.matrix(espana.train[, 2:94]),
    label = espana.train$score,
    nrounds = 1000,
    objective = "reg:squarederror",
    early_stopping_rounds = 3,
    max_depth = 6,
    eta = .25
      
  )

pred_xgb <- predict(spain.boost, data.matrix(espana.test[, 2:94]))

yhat <- pred_xgb
y <- espana.test$score
postResample(yhat, y)

      RMSE   Rsquared        MAE 
63.5236962  0.4392783 50.5827281 

r <- y - yhat
plot(r, ylab = "residuals", )

plot(y,
     yhat,
     xlab = "actual",
     ylab = "predicted",
     )
abline(lm(yhat ~ y))

#plot first 3 trees of model
xgb.plot.tree(model = spain.boost, trees = 0:2)

importance_matrix <- xgb.importance(model = spain.boost)
xgb.plot.importance(importance_matrix, xlab = "Feature Importance")

#grid search
#create hyperparameter grid
hyper_grid <- expand.grid(max_depth = seq(3, 6, 1),
                          eta = seq(.2, .35, .01))
xgb_train_rmse <- NULL
xgb_test_rmse <- NULL

for (j in 1:nrow(hyper_grid)) {
  set.seed(123)
  m_xgb_untuned <- xgb.cv(
    data = data.matrix(train[, 2:94]),
    label = train$score,
    nrounds = 1000,
    objective = "reg:squarederror",
    early_stopping_rounds = 3,
    nfold = 5,
    max_depth = hyper_grid$max_depth[j],
    eta = hyper_grid$eta[j]
  )
  
  xgb_train_rmse[j] <- m_xgb_untuned$evaluation_log$train_rmse_mean[m_xgb_untuned$best_iteration]
  xgb_test_rmse[j] <- m_xgb_untuned$evaluation_log$test_rmse_mean[m_xgb_untuned$best_iteration]
  
  cat(j, "\n")
}

#ideal hyperparamters
hyper_grid[which.min(xgb_test_rmse), ]

all.boost <- 
  
  xgboost(
    data = data.matrix(train[, 1:94]),
    label = train$score,
    nrounds = 1000,
    objective = "reg:squarederror",
    early_stopping_rounds = 3,
    max_depth = 6,
    eta = .25
  )

[1] train-rmse:370.803035 
Will train until train_rmse hasn't improved in 3 rounds.

[2] train-rmse:282.538115 
[3] train-rmse:217.469363 
[4] train-rmse:169.996374 
[5] train-rmse:135.902375 
[6] train-rmse:112.009252 
[7] train-rmse:95.687091 
[8] train-rmse:84.938868 
[9] train-rmse:77.943139 
[10]    train-rmse:73.380024 
[11]    train-rmse:70.593671 
[12]    train-rmse:68.691411 
[13]    train-rmse:67.448961 
[14]    train-rmse:66.542674 
[15]    train-rmse:65.893865 
[16]    train-rmse:65.359154 
[17]    train-rmse:64.938694 
[18]    train-rmse:64.574779 
[19]    train-rmse:64.252799 
[20]    train-rmse:63.938396 
[21]    train-rmse:63.646658 
[22]    train-rmse:63.274017 
[23]    train-rmse:63.025994 
[24]    train-rmse:62.819354 
[25]    train-rmse:62.633124 
[26]    train-rmse:62.450422 
[27]    train-rmse:62.199812 
[28]    train-rmse:62.047562 
[29]    train-rmse:61.843672 
[30]    train-rmse:61.605234 
[31]    train-rmse:61.214387 
[32]    train-rmse:61.068838 
[33]    train-rmse:60.912355 
[34]    train-rmse:60.786016 
[35]    train-rmse:60.661086 
[36]    train-rmse:60.535556 
[37]    train-rmse:60.420085 
[38]    train-rmse:60.299938 
[39]    train-rmse:60.119588 
[40]    train-rmse:60.012108 
[41]    train-rmse:59.830227 
[42]    train-rmse:59.727571 
[43]    train-rmse:59.618978 
[44]    train-rmse:59.475945 
[45]    train-rmse:59.383108 
[46]    train-rmse:59.278217 
[47]    train-rmse:59.177116 
[48]    train-rmse:59.082638 
[49]    train-rmse:58.963467 
[50]    train-rmse:58.865221 
[51]    train-rmse:58.793432 
[52]    train-rmse:58.709403 
[53]    train-rmse:58.634513 
[54]    train-rmse:58.549703 
[55]    train-rmse:58.371179 
[56]    train-rmse:58.288717 
[57]    train-rmse:58.197639 
[58]    train-rmse:58.113748 
[59]    train-rmse:58.039477 
[60]    train-rmse:57.973406 
[61]    train-rmse:57.865312 
[62]    train-rmse:57.787352 
[63]    train-rmse:57.691186 
[64]    train-rmse:57.611892 
[65]    train-rmse:57.547607 
[66]    train-rmse:57.456652 
[67]    train-rmse:57.383985 
[68]    train-rmse:57.325129 
[69]    train-rmse:57.247632 
[70]    train-rmse:57.191566 
[71]    train-rmse:57.129866 
[72]    train-rmse:57.064987 
[73]    train-rmse:57.007476 
[74]    train-rmse:56.942404 
[75]    train-rmse:56.837716 
[76]    train-rmse:56.757296 
[77]    train-rmse:56.684080 
[78]    train-rmse:56.615840 
[79]    train-rmse:56.574876 
[80]    train-rmse:56.514769 
[81]    train-rmse:56.457001 
[82]    train-rmse:56.403400 
[83]    train-rmse:56.330738 
[84]    train-rmse:56.291516 
[85]    train-rmse:56.234024 
[86]    train-rmse:56.168409 
[87]    train-rmse:56.098529 
[88]    train-rmse:56.041779 
[89]    train-rmse:55.985670 
[90]    train-rmse:55.924535 
[91]    train-rmse:55.888105 
[92]    train-rmse:55.836289 
[93]    train-rmse:55.778972 
[94]    train-rmse:55.723096 
[95]    train-rmse:55.669421 
[96]    train-rmse:55.609287 
[97]    train-rmse:55.550369 
[98]    train-rmse:55.496983 
[99]    train-rmse:55.445213 
[100]   train-rmse:55.396664 
[101]   train-rmse:55.350807 
[102]   train-rmse:55.325499 
[103]   train-rmse:55.282602 
[104]   train-rmse:55.229428 
[105]   train-rmse:55.177941 
[106]   train-rmse:55.124717 
[107]   train-rmse:55.041684 
[108]   train-rmse:55.003039 
[109]   train-rmse:54.948750 
[110]   train-rmse:54.898064 
[111]   train-rmse:54.857512 
[112]   train-rmse:54.806652 
[113]   train-rmse:54.759340 
[114]   train-rmse:54.713403 
[115]   train-rmse:54.679029 
[116]   train-rmse:54.632848 
[117]   train-rmse:54.586188 
[118]   train-rmse:54.546852 
[119]   train-rmse:54.490593 
[120]   train-rmse:54.443871 
[121]   train-rmse:54.395245 
[122]   train-rmse:54.352873 
[123]   train-rmse:54.307384 
[124]   train-rmse:54.259725 
[125]   train-rmse:54.221768 
[126]   train-rmse:54.172678 
[127]   train-rmse:54.122063 
[128]   train-rmse:54.089019 
[129]   train-rmse:54.038328 
[130]   train-rmse:54.002211 
[131]   train-rmse:53.948297 
[132]   train-rmse:53.910653 
[133]   train-rmse:53.872455 
[134]   train-rmse:53.825612 
[135]   train-rmse:53.801749 
[136]   train-rmse:53.761652 
[137]   train-rmse:53.701094 
[138]   train-rmse:53.660834 
[139]   train-rmse:53.616470 
[140]   train-rmse:53.551472 
[141]   train-rmse:53.498767 
[142]   train-rmse:53.454650 
[143]   train-rmse:53.399845 
[144]   train-rmse:53.360469 
[145]   train-rmse:53.323903 
[146]   train-rmse:53.285532 
[147]   train-rmse:53.240054 
[148]   train-rmse:53.211420 
[149]   train-rmse:53.197898 
[150]   train-rmse:53.184605 
[151]   train-rmse:53.148796 
[152]   train-rmse:53.099326 
[153]   train-rmse:53.059075 
[154]   train-rmse:53.017557 
[155]   train-rmse:52.966475 
[156]   train-rmse:52.929526 
[157]   train-rmse:52.890055 
[158]   train-rmse:52.853125 
[159]   train-rmse:52.810778 
[160]   train-rmse:52.768234 
[161]   train-rmse:52.728144 
[162]   train-rmse:52.688333 
[163]   train-rmse:52.650422 
[164]   train-rmse:52.615036 
[165]   train-rmse:52.575466 
[166]   train-rmse:52.535043 
[167]   train-rmse:52.494245 
[168]   train-rmse:52.451695 
[169]   train-rmse:52.410434 
[170]   train-rmse:52.386551 
[171]   train-rmse:52.351752 
[172]   train-rmse:52.309898 
[173]   train-rmse:52.268816 
[174]   train-rmse:52.229144 
[175]   train-rmse:52.191127 
[176]   train-rmse:52.153328 
[177]   train-rmse:52.116287 
[178]   train-rmse:52.076457 
[179]   train-rmse:52.035351 
[180]   train-rmse:51.998065 
[181]   train-rmse:51.968279 
[182]   train-rmse:51.925861 
[183]   train-rmse:51.889835 
[184]   train-rmse:51.855458 
[185]   train-rmse:51.807316 
[186]   train-rmse:51.773978 
[187]   train-rmse:51.743478 
[188]   train-rmse:51.724441 
[189]   train-rmse:51.694691 
[190]   train-rmse:51.656644 
[191]   train-rmse:51.627641 
[192]   train-rmse:51.586368 
[193]   train-rmse:51.547628 
[194]   train-rmse:51.516709 
[195]   train-rmse:51.484049 
[196]   train-rmse:51.436085 
[197]   train-rmse:51.399907 
[198]   train-rmse:51.362481 
[199]   train-rmse:51.346789 
[200]   train-rmse:51.319289 
[201]   train-rmse:51.278258 
[202]   train-rmse:51.241755 
[203]   train-rmse:51.209173 
[204]   train-rmse:51.154106 
[205]   train-rmse:51.124680 
[206]   train-rmse:51.082823 
[207]   train-rmse:51.048545 
[208]   train-rmse:51.006701 
[209]   train-rmse:50.965312 
[210]   train-rmse:50.931472 
[211]   train-rmse:50.910324 
[212]   train-rmse:50.875935 
[213]   train-rmse:50.845053 
[214]   train-rmse:50.811642 
[215]   train-rmse:50.792099 
[216]   train-rmse:50.754722 
[217]   train-rmse:50.721669 
[218]   train-rmse:50.688209 
[219]   train-rmse:50.680042 
[220]   train-rmse:50.652909 
[221]   train-rmse:50.608968 
[222]   train-rmse:50.588873 
[223]   train-rmse:50.559553 
[224]   train-rmse:50.539646 
[225]   train-rmse:50.500038 
[226]   train-rmse:50.478380 
[227]   train-rmse:50.446129 
[228]   train-rmse:50.413158 
[229]   train-rmse:50.375950 
[230]   train-rmse:50.345580 
[231]   train-rmse:50.323545 
[232]   train-rmse:50.294164 
[233]   train-rmse:50.264242 
[234]   train-rmse:50.236484 
[235]   train-rmse:50.209348 
[236]   train-rmse:50.188450 
[237]   train-rmse:50.162262 
[238]   train-rmse:50.119152 
[239]   train-rmse:50.083078 
[240]   train-rmse:50.054170 
[241]   train-rmse:50.018757 
[242]   train-rmse:49.991178 
[243]   train-rmse:49.956426 
[244]   train-rmse:49.929188 
[245]   train-rmse:49.896266 
[246]   train-rmse:49.868926 
[247]   train-rmse:49.840152 
[248]   train-rmse:49.810138 
[249]   train-rmse:49.782767 
[250]   train-rmse:49.741961 
[251]   train-rmse:49.709980 
[252]   train-rmse:49.678187 
[253]   train-rmse:49.654226 
[254]   train-rmse:49.624483 
[255]   train-rmse:49.587674 
[256]   train-rmse:49.562836 
[257]   train-rmse:49.537432 
[258]   train-rmse:49.500555 
[259]   train-rmse:49.473811 
[260]   train-rmse:49.447171 
[261]   train-rmse:49.426744 
[262]   train-rmse:49.394609 
[263]   train-rmse:49.368202 
[264]   train-rmse:49.335362 
[265]   train-rmse:49.301916 
[266]   train-rmse:49.267883 
[267]   train-rmse:49.238375 
[268]   train-rmse:49.204305 
[269]   train-rmse:49.170559 
[270]   train-rmse:49.132636 
[271]   train-rmse:49.100834 
[272]   train-rmse:49.078901 
[273]   train-rmse:49.045537 
[274]   train-rmse:49.012628 
[275]   train-rmse:48.981001 
[276]   train-rmse:48.949943 
[277]   train-rmse:48.918525 
[278]   train-rmse:48.883564 
[279]   train-rmse:48.853557 
[280]   train-rmse:48.819374 
[281]   train-rmse:48.784237 
[282]   train-rmse:48.754307 
[283]   train-rmse:48.730377 
[284]   train-rmse:48.714140 
[285]   train-rmse:48.683774 
[286]   train-rmse:48.658498 
[287]   train-rmse:48.622691 
[288]   train-rmse:48.595229 
[289]   train-rmse:48.560735 
[290]   train-rmse:48.539718 
[291]   train-rmse:48.510591 
[292]   train-rmse:48.488644 
[293]   train-rmse:48.458187 
[294]   train-rmse:48.428475 
[295]   train-rmse:48.395611 
[296]   train-rmse:48.371707 
[297]   train-rmse:48.348815 
[298]   train-rmse:48.317085 
[299]   train-rmse:48.292797 
[300]   train-rmse:48.260280 
[301]   train-rmse:48.235408 
[302]   train-rmse:48.220517 
[303]   train-rmse:48.188759 
[304]   train-rmse:48.159555 
[305]   train-rmse:48.130495 
[306]   train-rmse:48.102870 
[307]   train-rmse:48.063439 
[308]   train-rmse:48.049884 
[309]   train-rmse:48.020183 
[310]   train-rmse:48.001217 
[311]   train-rmse:47.963656 
[312]   train-rmse:47.939155 
[313]   train-rmse:47.911671 
[314]   train-rmse:47.880059 
[315]   train-rmse:47.849274 
[316]   train-rmse:47.822918 
[317]   train-rmse:47.799423 
[318]   train-rmse:47.770137 
[319]   train-rmse:47.735567 
[320]   train-rmse:47.708550 
[321]   train-rmse:47.674455 
[322]   train-rmse:47.648633 
[323]   train-rmse:47.629243 
[324]   train-rmse:47.596334 
[325]   train-rmse:47.565312 
[326]   train-rmse:47.547230 
[327]   train-rmse:47.515190 
[328]   train-rmse:47.497805 
[329]   train-rmse:47.474955 
[330]   train-rmse:47.459329 
[331]   train-rmse:47.433642 
[332]   train-rmse:47.410852 
[333]   train-rmse:47.381203 
[334]   train-rmse:47.355782 
[335]   train-rmse:47.330883 
[336]   train-rmse:47.305679 
[337]   train-rmse:47.285632 
[338]   train-rmse:47.254717 
[339]   train-rmse:47.229280 
[340]   train-rmse:47.206210 
[341]   train-rmse:47.176984 
[342]   train-rmse:47.145298 
[343]   train-rmse:47.118714 
[344]   train-rmse:47.108438 
[345]   train-rmse:47.081507 
[346]   train-rmse:47.058825 
[347]   train-rmse:47.040950 
[348]   train-rmse:47.019884 
[349]   train-rmse:46.995899 
[350]   train-rmse:46.968957 
[351]   train-rmse:46.951820 
[352]   train-rmse:46.924345 
[353]   train-rmse:46.896445 
[354]   train-rmse:46.869659 
[355]   train-rmse:46.843792 
[356]   train-rmse:46.815716 
[357]   train-rmse:46.794883 
[358]   train-rmse:46.769581 
[359]   train-rmse:46.744741 
[360]   train-rmse:46.716991 
[361]   train-rmse:46.693077 
[362]   train-rmse:46.660182 
[363]   train-rmse:46.633224 
[364]   train-rmse:46.605196 
[365]   train-rmse:46.585076 
[366]   train-rmse:46.566576 
[367]   train-rmse:46.538388 
[368]   train-rmse:46.510704 
[369]   train-rmse:46.496810 
[370]   train-rmse:46.467335 
[371]   train-rmse:46.438312 
[372]   train-rmse:46.414201 
[373]   train-rmse:46.385331 
[374]   train-rmse:46.355695 
[375]   train-rmse:46.331765 
[376]   train-rmse:46.306277 
[377]   train-rmse:46.283103 
[378]   train-rmse:46.265289 
[379]   train-rmse:46.239542 
[380]   train-rmse:46.216530 
[381]   train-rmse:46.188877 
[382]   train-rmse:46.162253 
[383]   train-rmse:46.134009 
[384]   train-rmse:46.107054 
[385]   train-rmse:46.082358 
[386]   train-rmse:46.057430 
[387]   train-rmse:46.037155 
[388]   train-rmse:46.019194 
[389]   train-rmse:45.987464 
[390]   train-rmse:45.964339 
[391]   train-rmse:45.935811 
[392]   train-rmse:45.907364 
[393]   train-rmse:45.881531 
[394]   train-rmse:45.854972 
[395]   train-rmse:45.827679 
[396]   train-rmse:45.805567 
[397]   train-rmse:45.786536 
[398]   train-rmse:45.765688 
[399]   train-rmse:45.739748 
[400]   train-rmse:45.712985 
[401]   train-rmse:45.699824 
[402]   train-rmse:45.660898 
[403]   train-rmse:45.636136 
[404]   train-rmse:45.605763 
[405]   train-rmse:45.581222 
[406]   train-rmse:45.559790 
[407]   train-rmse:45.536872 
[408]   train-rmse:45.515688 
[409]   train-rmse:45.495719 
[410]   train-rmse:45.471847 
[411]   train-rmse:45.458532 
[412]   train-rmse:45.439776 
[413]   train-rmse:45.417097 
[414]   train-rmse:45.390302 
[415]   train-rmse:45.367569 
[416]   train-rmse:45.355267 
[417]   train-rmse:45.332624 
[418]   train-rmse:45.310042 
[419]   train-rmse:45.287100 
[420]   train-rmse:45.260996 
[421]   train-rmse:45.237497 
[422]   train-rmse:45.227843 
[423]   train-rmse:45.216894 
[424]   train-rmse:45.197858 
[425]   train-rmse:45.169280 
[426]   train-rmse:45.140741 
[427]   train-rmse:45.111274 
[428]   train-rmse:45.085964 
[429]   train-rmse:45.058958 
[430]   train-rmse:45.047042 
[431]   train-rmse:45.021497 
[432]   train-rmse:45.000772 
[433]   train-rmse:44.976720 
[434]   train-rmse:44.953505 
[435]   train-rmse:44.936402 
[436]   train-rmse:44.915188 
[437]   train-rmse:44.901255 
[438]   train-rmse:44.874428 
[439]   train-rmse:44.851361 
[440]   train-rmse:44.828067 
[441]   train-rmse:44.805159 
[442]   train-rmse:44.776850 
[443]   train-rmse:44.746089 
[444]   train-rmse:44.732443 
[445]   train-rmse:44.722435 
[446]   train-rmse:44.701350 
[447]   train-rmse:44.681939 
[448]   train-rmse:44.672167 
[449]   train-rmse:44.638475 
[450]   train-rmse:44.610636 
[451]   train-rmse:44.600605 
[452]   train-rmse:44.573948 
[453]   train-rmse:44.553312 
[454]   train-rmse:44.525413 
[455]   train-rmse:44.511232 
[456]   train-rmse:44.494876 
[457]   train-rmse:44.466005 
[458]   train-rmse:44.441022 
[459]   train-rmse:44.419128 
[460]   train-rmse:44.391561 
[461]   train-rmse:44.364777 
[462]   train-rmse:44.344953 
[463]   train-rmse:44.317777 
[464]   train-rmse:44.292523 
[465]   train-rmse:44.266136 
[466]   train-rmse:44.245415 
[467]   train-rmse:44.217020 
[468]   train-rmse:44.191122 
[469]   train-rmse:44.162760 
[470]   train-rmse:44.139471 
[471]   train-rmse:44.116839 
[472]   train-rmse:44.098602 
[473]   train-rmse:44.079014 
[474]   train-rmse:44.050159 
[475]   train-rmse:44.027052 
[476]   train-rmse:43.991073 
[477]   train-rmse:43.970810 
[478]   train-rmse:43.951508 
[479]   train-rmse:43.933463 
[480]   train-rmse:43.918117 
[481]   train-rmse:43.909142 
[482]   train-rmse:43.883313 
[483]   train-rmse:43.858001 
[484]   train-rmse:43.839768 
[485]   train-rmse:43.819684 
[486]   train-rmse:43.793740 
[487]   train-rmse:43.769767 
[488]   train-rmse:43.744652 
[489]   train-rmse:43.736473 
[490]   train-rmse:43.726047 
[491]   train-rmse:43.710812 
[492]   train-rmse:43.686606 
[493]   train-rmse:43.660931 
[494]   train-rmse:43.637587 
[495]   train-rmse:43.628376 
[496]   train-rmse:43.604982 
[497]   train-rmse:43.579097 
[498]   train-rmse:43.556273 
[499]   train-rmse:43.535220 
[500]   train-rmse:43.512601 
[501]   train-rmse:43.491757 
[502]   train-rmse:43.466949 
[503]   train-rmse:43.457427 
[504]   train-rmse:43.435905 
[505]   train-rmse:43.411804 
[506]   train-rmse:43.389811 
[507]   train-rmse:43.370265 
[508]   train-rmse:43.352014 
[509]   train-rmse:43.341935 
[510]   train-rmse:43.317784 
[511]   train-rmse:43.292464 
[512]   train-rmse:43.265868 
[513]   train-rmse:43.244434 
[514]   train-rmse:43.215442 
[515]   train-rmse:43.193756 
[516]   train-rmse:43.174192 
[517]   train-rmse:43.150389 
[518]   train-rmse:43.144678 
[519]   train-rmse:43.123216 
[520]   train-rmse:43.114122 
[521]   train-rmse:43.093833 
[522]   train-rmse:43.070501 
[523]   train-rmse:43.053871 
[524]   train-rmse:43.028302 
[525]   train-rmse:43.002784 
[526]   train-rmse:42.979678 
[527]   train-rmse:42.968602 
[528]   train-rmse:42.946392 
[529]   train-rmse:42.941421 
[530]   train-rmse:42.925194 
[531]   train-rmse:42.912718 
[532]   train-rmse:42.895070 
[533]   train-rmse:42.870678 
[534]   train-rmse:42.852834 
[535]   train-rmse:42.831289 
[536]   train-rmse:42.807822 
[537]   train-rmse:42.794194 
[538]   train-rmse:42.771248 
[539]   train-rmse:42.756022 
[540]   train-rmse:42.739542 
[541]   train-rmse:42.715877 
[542]   train-rmse:42.695098 
[543]   train-rmse:42.684590 
[544]   train-rmse:42.666602 
[545]   train-rmse:42.647175 
[546]   train-rmse:42.628339 
[547]   train-rmse:42.612945 
[548]   train-rmse:42.587920 
[549]   train-rmse:42.571635 
[550]   train-rmse:42.556547 
[551]   train-rmse:42.545365 
[552]   train-rmse:42.523135 
[553]   train-rmse:42.499429 
[554]   train-rmse:42.474477 
[555]   train-rmse:42.450296 
[556]   train-rmse:42.428811 
[557]   train-rmse:42.424900 
[558]   train-rmse:42.418562 
[559]   train-rmse:42.392943 
[560]   train-rmse:42.376112 
[561]   train-rmse:42.354374 
[562]   train-rmse:42.331156 
[563]   train-rmse:42.321814 
[564]   train-rmse:42.302320 
[565]   train-rmse:42.291076 
[566]   train-rmse:42.269660 
[567]   train-rmse:42.245724 
[568]   train-rmse:42.233591 
[569]   train-rmse:42.214124 
[570]   train-rmse:42.188596 
[571]   train-rmse:42.165838 
[572]   train-rmse:42.149579 
[573]   train-rmse:42.122555 
[574]   train-rmse:42.098405 
[575]   train-rmse:42.083283 
[576]   train-rmse:42.056935 
[577]   train-rmse:42.032460 
[578]   train-rmse:42.012596 
[579]   train-rmse:41.987041 
[580]   train-rmse:41.962819 
[581]   train-rmse:41.946734 
[582]   train-rmse:41.917388 
[583]   train-rmse:41.896108 
[584]   train-rmse:41.880038 
[585]   train-rmse:41.864266 
[586]   train-rmse:41.845268 
[587]   train-rmse:41.821098 
[588]   train-rmse:41.799215 
[589]   train-rmse:41.779613 
[590]   train-rmse:41.768436 
[591]   train-rmse:41.754211 
[592]   train-rmse:41.734460 
[593]   train-rmse:41.710531 
[594]   train-rmse:41.697114 
[595]   train-rmse:41.678632 
[596]   train-rmse:41.659520 
[597]   train-rmse:41.643250 
[598]   train-rmse:41.619598 
[599]   train-rmse:41.601932 
[600]   train-rmse:41.590098 
[601]   train-rmse:41.565588 
[602]   train-rmse:41.552112 
[603]   train-rmse:41.531911 
[604]   train-rmse:41.507599 
[605]   train-rmse:41.484976 
[606]   train-rmse:41.472694 
[607]   train-rmse:41.445920 
[608]   train-rmse:41.423629 
[609]   train-rmse:41.406608 
[610]   train-rmse:41.394465 
[611]   train-rmse:41.374112 
[612]   train-rmse:41.349376 
[613]   train-rmse:41.327039 
[614]   train-rmse:41.315647 
[615]   train-rmse:41.294696 
[616]   train-rmse:41.273460 
[617]   train-rmse:41.260966 
[618]   train-rmse:41.239491 
[619]   train-rmse:41.236010 
[620]   train-rmse:41.224524 
[621]   train-rmse:41.206804 
[622]   train-rmse:41.188412 
[623]   train-rmse:41.170522 
[624]   train-rmse:41.146562 
[625]   train-rmse:41.121045 
[626]   train-rmse:41.098936 
[627]   train-rmse:41.076377 
[628]   train-rmse:41.055132 
[629]   train-rmse:41.034964 
[630]   train-rmse:41.013830 
[631]   train-rmse:40.988542 
[632]   train-rmse:40.968723 
[633]   train-rmse:40.943848 
[634]   train-rmse:40.918710 
[635]   train-rmse:40.896451 
[636]   train-rmse:40.870059 
[637]   train-rmse:40.851513 
[638]   train-rmse:40.830849 
[639]   train-rmse:40.808666 
[640]   train-rmse:40.786134 
[641]   train-rmse:40.764081 
[642]   train-rmse:40.743483 
[643]   train-rmse:40.717164 
[644]   train-rmse:40.709013 
[645]   train-rmse:40.688317 
[646]   train-rmse:40.676021 
[647]   train-rmse:40.654077 
[648]   train-rmse:40.630465 
[649]   train-rmse:40.605037 
[650]   train-rmse:40.584059 
[651]   train-rmse:40.564706 
[652]   train-rmse:40.541381 
[653]   train-rmse:40.524074 
[654]   train-rmse:40.515809 
[655]   train-rmse:40.496885 
[656]   train-rmse:40.476281 
[657]   train-rmse:40.456411 
[658]   train-rmse:40.438201 
[659]   train-rmse:40.420556 
[660]   train-rmse:40.400815 
[661]   train-rmse:40.387685 
[662]   train-rmse:40.364529 
[663]   train-rmse:40.341238 
[664]   train-rmse:40.316650 
[665]   train-rmse:40.300294 
[666]   train-rmse:40.275438 
[667]   train-rmse:40.256346 
[668]   train-rmse:40.239737 
[669]   train-rmse:40.219719 
[670]   train-rmse:40.200354 
[671]   train-rmse:40.185967 
[672]   train-rmse:40.164385 
[673]   train-rmse:40.150778 
[674]   train-rmse:40.132435 
[675]   train-rmse:40.112340 
[676]   train-rmse:40.093360 
[677]   train-rmse:40.069534 
[678]   train-rmse:40.057672 
[679]   train-rmse:40.041621 
[680]   train-rmse:40.018957 
[681]   train-rmse:39.999130 
[682]   train-rmse:39.973812 
[683]   train-rmse:39.950951 
[684]   train-rmse:39.926967 
[685]   train-rmse:39.904950 
[686]   train-rmse:39.883734 
[687]   train-rmse:39.864243 
[688]   train-rmse:39.847163 
[689]   train-rmse:39.826504 
[690]   train-rmse:39.807666 
[691]   train-rmse:39.790308 
[692]   train-rmse:39.768259 
[693]   train-rmse:39.761889 
[694]   train-rmse:39.748081 
[695]   train-rmse:39.726135 
[696]   train-rmse:39.705801 
[697]   train-rmse:39.683978 
[698]   train-rmse:39.661964 
[699]   train-rmse:39.649972 
[700]   train-rmse:39.644628 
[701]   train-rmse:39.642131 
[702]   train-rmse:39.622009 
[703]   train-rmse:39.612676 
[704]   train-rmse:39.591405 
[705]   train-rmse:39.574780 
[706]   train-rmse:39.557883 
[707]   train-rmse:39.549113 
[708]   train-rmse:39.530127 
[709]   train-rmse:39.507836 
[710]   train-rmse:39.488358 
[711]   train-rmse:39.472363 
[712]   train-rmse:39.453227 
[713]   train-rmse:39.437114 
[714]   train-rmse:39.433297 
[715]   train-rmse:39.411974 
[716]   train-rmse:39.401711 
[717]   train-rmse:39.390351 
[718]   train-rmse:39.370278 
[719]   train-rmse:39.350988 
[720]   train-rmse:39.327934 
[721]   train-rmse:39.305177 
[722]   train-rmse:39.289607 
[723]   train-rmse:39.278089 
[724]   train-rmse:39.256881 
[725]   train-rmse:39.238886 
[726]   train-rmse:39.218077 
[727]   train-rmse:39.210946 
[728]   train-rmse:39.188707 
[729]   train-rmse:39.170558 
[730]   train-rmse:39.154652 
[731]   train-rmse:39.130860 
[732]   train-rmse:39.121586 
[733]   train-rmse:39.108651 
[734]   train-rmse:39.092016 
[735]   train-rmse:39.069678 
[736]   train-rmse:39.051864 
[737]   train-rmse:39.038329 
[738]   train-rmse:39.016915 
[739]   train-rmse:38.995638 
[740]   train-rmse:38.977667 
[741]   train-rmse:38.955607 
[742]   train-rmse:38.938619 
[743]   train-rmse:38.920564 
[744]   train-rmse:38.906952 
[745]   train-rmse:38.901799 
[746]   train-rmse:38.891323 
[747]   train-rmse:38.870315 
[748]   train-rmse:38.860192 
[749]   train-rmse:38.841358 
[750]   train-rmse:38.824077 
[751]   train-rmse:38.804859 
[752]   train-rmse:38.781540 
[753]   train-rmse:38.760779 
[754]   train-rmse:38.737830 
[755]   train-rmse:38.719492 
[756]   train-rmse:38.697916 
[757]   train-rmse:38.692169 
[758]   train-rmse:38.677351 
[759]   train-rmse:38.656274 
[760]   train-rmse:38.636101 
[761]   train-rmse:38.616284 
[762]   train-rmse:38.599948 
[763]   train-rmse:38.585672 
[764]   train-rmse:38.567184 
[765]   train-rmse:38.561966 
[766]   train-rmse:38.550132 
[767]   train-rmse:38.542071 
[768]   train-rmse:38.519721 
[769]   train-rmse:38.503053 
[770]   train-rmse:38.482223 
[771]   train-rmse:38.466734 
[772]   train-rmse:38.449587 
[773]   train-rmse:38.434166 
[774]   train-rmse:38.415706 
[775]   train-rmse:38.393912 
[776]   train-rmse:38.376913 
[777]   train-rmse:38.359604 
[778]   train-rmse:38.339955 
[779]   train-rmse:38.317074 
[780]   train-rmse:38.300270 
[781]   train-rmse:38.292575 
[782]   train-rmse:38.279920 
[783]   train-rmse:38.268869 
[784]   train-rmse:38.252591 
[785]   train-rmse:38.241380 
[786]   train-rmse:38.232361 
[787]   train-rmse:38.215563 
[788]   train-rmse:38.194283 
[789]   train-rmse:38.174412 
[790]   train-rmse:38.152277 
[791]   train-rmse:38.130020 
[792]   train-rmse:38.113191 
[793]   train-rmse:38.091825 
[794]   train-rmse:38.072699 
[795]   train-rmse:38.053873 
[796]   train-rmse:38.034660 
[797]   train-rmse:38.016120 
[798]   train-rmse:38.003502 
[799]   train-rmse:37.985381 
[800]   train-rmse:37.970161 
[801]   train-rmse:37.953374 
[802]   train-rmse:37.946186 
[803]   train-rmse:37.928264 
[804]   train-rmse:37.919100 
[805]   train-rmse:37.900212 
[806]   train-rmse:37.880693 
[807]   train-rmse:37.863209 
[808]   train-rmse:37.845672 
[809]   train-rmse:37.832251 
[810]   train-rmse:37.809978 
[811]   train-rmse:37.791829 
[812]   train-rmse:37.779271 
[813]   train-rmse:37.760066 
[814]   train-rmse:37.741885 
[815]   train-rmse:37.723065 
[816]   train-rmse:37.717768 
[817]   train-rmse:37.711824 
[818]   train-rmse:37.705755 
[819]   train-rmse:37.690312 
[820]   train-rmse:37.671963 
[821]   train-rmse:37.654479 
[822]   train-rmse:37.638956 
[823]   train-rmse:37.621439 
[824]   train-rmse:37.602864 
[825]   train-rmse:37.587849 
[826]   train-rmse:37.574504 
[827]   train-rmse:37.559308 
[828]   train-rmse:37.538401 
[829]   train-rmse:37.535555 
[830]   train-rmse:37.532042 
[831]   train-rmse:37.519115 
[832]   train-rmse:37.497706 
[833]   train-rmse:37.473016 
[834]   train-rmse:37.455350 
[835]   train-rmse:37.438290 
[836]   train-rmse:37.423109 
[837]   train-rmse:37.410197 
[838]   train-rmse:37.403237 
[839]   train-rmse:37.386922 
[840]   train-rmse:37.365130 
[841]   train-rmse:37.349127 
[842]   train-rmse:37.335020 
[843]   train-rmse:37.313051 
[844]   train-rmse:37.296105 
[845]   train-rmse:37.283767 
[846]   train-rmse:37.263269 
[847]   train-rmse:37.250499 
[848]   train-rmse:37.238361 
[849]   train-rmse:37.219384 
[850]   train-rmse:37.203282 
[851]   train-rmse:37.189562 
[852]   train-rmse:37.181153 
[853]   train-rmse:37.165576 
[854]   train-rmse:37.149238 
[855]   train-rmse:37.133276 
[856]   train-rmse:37.123502 
[857]   train-rmse:37.111751 
[858]   train-rmse:37.107397 
[859]   train-rmse:37.097317 
[860]   train-rmse:37.080511 
[861]   train-rmse:37.065869 
[862]   train-rmse:37.054739 
[863]   train-rmse:37.038985 
[864]   train-rmse:37.031322 
[865]   train-rmse:37.018071 
[866]   train-rmse:36.998935 
[867]   train-rmse:36.989487 
[868]   train-rmse:36.972072 
[869]   train-rmse:36.956987 
[870]   train-rmse:36.947453 
[871]   train-rmse:36.941184 
[872]   train-rmse:36.924650 
[873]   train-rmse:36.904634 
[874]   train-rmse:36.896120 
[875]   train-rmse:36.878703 
[876]   train-rmse:36.861500 
[877]   train-rmse:36.844450 
[878]   train-rmse:36.824296 
[879]   train-rmse:36.802889 
[880]   train-rmse:36.786294 
[881]   train-rmse:36.766853 
[882]   train-rmse:36.750349 
[883]   train-rmse:36.737097 
[884]   train-rmse:36.733313 
[885]   train-rmse:36.722477 
[886]   train-rmse:36.707087 
[887]   train-rmse:36.689311 
[888]   train-rmse:36.668770 
[889]   train-rmse:36.656029 
[890]   train-rmse:36.639427 
[891]   train-rmse:36.621083 
[892]   train-rmse:36.603130 
[893]   train-rmse:36.585655 
[894]   train-rmse:36.569519 
[895]   train-rmse:36.549252 
[896]   train-rmse:36.540341 
[897]   train-rmse:36.533740 
[898]   train-rmse:36.518341 
[899]   train-rmse:36.502966 
[900]   train-rmse:36.489497 
[901]   train-rmse:36.476702 
[902]   train-rmse:36.462063 
[903]   train-rmse:36.452338 
[904]   train-rmse:36.433404 
[905]   train-rmse:36.414264 
[906]   train-rmse:36.396751 
[907]   train-rmse:36.377826 
[908]   train-rmse:36.367692 
[909]   train-rmse:36.348750 
[910]   train-rmse:36.330215 
[911]   train-rmse:36.313528 
[912]   train-rmse:36.297126 
[913]   train-rmse:36.279003 
[914]   train-rmse:36.258874 
[915]   train-rmse:36.246217 
[916]   train-rmse:36.227176 
[917]   train-rmse:36.212236 
[918]   train-rmse:36.193580 
[919]   train-rmse:36.172662 
[920]   train-rmse:36.160423 
[921]   train-rmse:36.146418 
[922]   train-rmse:36.138124 
[923]   train-rmse:36.123746 
[924]   train-rmse:36.114528 
[925]   train-rmse:36.096257 
[926]   train-rmse:36.077179 
[927]   train-rmse:36.061042 
[928]   train-rmse:36.047146 
[929]   train-rmse:36.035905 
[930]   train-rmse:36.030216 
[931]   train-rmse:36.023702 
[932]   train-rmse:36.014739 
[933]   train-rmse:35.998817 
[934]   train-rmse:35.993595 
[935]   train-rmse:35.983263 
[936]   train-rmse:35.975335 
[937]   train-rmse:35.964748 
[938]   train-rmse:35.947927 
[939]   train-rmse:35.939715 
[940]   train-rmse:35.925603 
[941]   train-rmse:35.906594 
[942]   train-rmse:35.895803 
[943]   train-rmse:35.891496 
[944]   train-rmse:35.874832 
[945]   train-rmse:35.857372 
[946]   train-rmse:35.842795 
[947]   train-rmse:35.827460 
[948]   train-rmse:35.808954 
[949]   train-rmse:35.790656 
[950]   train-rmse:35.775075 
[951]   train-rmse:35.761179 
[952]   train-rmse:35.757741 
[953]   train-rmse:35.742927 
[954]   train-rmse:35.729886 
[955]   train-rmse:35.719827 
[956]   train-rmse:35.706743 
[957]   train-rmse:35.689770 
[958]   train-rmse:35.676942 
[959]   train-rmse:35.663891 
[960]   train-rmse:35.649042 
[961]   train-rmse:35.633660 
[962]   train-rmse:35.618079 
[963]   train-rmse:35.602022 
[964]   train-rmse:35.580306 
[965]   train-rmse:35.566798 
[966]   train-rmse:35.548080 
[967]   train-rmse:35.530461 
[968]   train-rmse:35.519280 
[969]   train-rmse:35.502069 
[970]   train-rmse:35.484000 
[971]   train-rmse:35.465055 
[972]   train-rmse:35.447522 
[973]   train-rmse:35.430962 
[974]   train-rmse:35.417981 
[975]   train-rmse:35.397031 
[976]   train-rmse:35.384413 
[977]   train-rmse:35.365904 
[978]   train-rmse:35.355714 
[979]   train-rmse:35.344873 
[980]   train-rmse:35.330202 
[981]   train-rmse:35.314881 
[982]   train-rmse:35.299530 
[983]   train-rmse:35.281390 
[984]   train-rmse:35.264178 
[985]   train-rmse:35.252101 
[986]   train-rmse:35.239490 
[987]   train-rmse:35.222099 
[988]   train-rmse:35.209109 
[989]   train-rmse:35.203801 
[990]   train-rmse:35.194536 
[991]   train-rmse:35.179364 
[992]   train-rmse:35.168752 
[993]   train-rmse:35.161019 
[994]   train-rmse:35.142173 
[995]   train-rmse:35.130742 
[996]   train-rmse:35.119604 
[997]   train-rmse:35.100532 
[998]   train-rmse:35.082863 
[999]   train-rmse:35.068889 
[1000]  train-rmse:35.053877 

pred_xgb.all <- predict(all.boost, data.matrix(test[, 1:94]))

yhat <- pred_xgb.all
y <- test$score
postResample(yhat, y)

      RMSE   Rsquared        MAE 
60.1041069  0.5843453 47.4517182 

r <- y - yhat
plot(r, ylab = "residuals", )

plot(y,
     yhat,
     xlab = "actual",
     ylab = "predicted",
     )
abline(lm(yhat ~ y))

#plot first 3 trees of model
xgb.plot.tree(model = all.boost, trees = 0:2)

importance_matrix <- xgb.importance(model = all.boost)
xgb.plot.importance(importance_matrix, xlab = "Feature Importance")

pred.train <- predict(espana.model, newdata=espana.train) #R2 train
pred.test <- predict(espana.model, newdata=espana.test)   #R2 test

lin.reg.R2.espana.train <- R2(pred.train,espana.train$score)
lin.reg.MAE.espana.train <-mean(abs(espana.train$score-pred.train))
lin.reg.RMSE.espana.train <-sqrt(mean((espana.train$score-pred.train)^2))

lin.reg.R2.espana.test <- R2(pred.test,espana.test$score)                 
lin.reg.MAE.espana.test <-mean(abs(espana.test$score-pred.test))
lin.reg.RMSE.espana.test <-sqrt(mean((espana.test$score-pred.test)^2))

linear.summary <- data.frame(
             train.r2 = lin.reg.R2.espana.train,
             train.MAE = lin.reg.MAE.espana.train,
             train.RMSE = lin.reg.RMSE.espana.train,
             test.r2 = lin.reg.R2.espana.test,
             test.MAE = lin.reg.MAE.espana.test,
             test.RMSE = lin.reg.RMSE.espana.test )

print(linear.summary)
             

#predict cart
cart.espana.train.pred <- predict(best.tree, newdata =espana.train )
cart.espana.test.pred <- predict(best.tree, newdata = espana.test)


CART.R2.espana.train <- R2(cart.espana.train.pred,espana.train$score)
CART.MAE.espana.train <- mean(abs(espana.train$score-cart.espana.train.pred))
CART.RMSE.espana.train <- sqrt(mean((espana.train$score-cart.espana.train.pred)^2))

CART.R2.espana.test <- R2(cart.espana.test.pred,espana.test$score)
CART.MAE.espana.test <- mean(abs(espana.test$score-cart.espana.test.pred))
CART.RMSE.espana.test <- sqrt(mean((espana.test$score-cart.espana.test.pred)^2))

#predict RF

rf.espana.train.pred <- predict(best.rf,newdata = espana.train)
rf.espana.test.pred <- predict(best.rf,newdata = espana.test)

RF.R2.espana.train <- R2(rf.espana.train.pred,espana.train$score)
RF.MAE.espana.train <- mean(abs(espana.train$score-rf.espana.train.pred))
RF.RMSE.espana.train <- sqrt(mean((espana.train$score-rf.espana.train.pred)^2))


RF.R2.espana.test <- R2(rf.espana.test.pred,espana.test$score)
RF.MAE.espana.test <- mean(abs(espana.test$score-rf.espana.test.pred))
RF.RMSE.espana.test <- sqrt(mean((espana.test$score-rf.espana.test.pred)^2))

# Summary

a = "Linear"
b = "Cart"
c = "Rf"
summary_statistics <- data.frame(
  row.name = c(a,b,c),
  IS.R2 = c(lin.reg.R2.espana.train,CART.R2.espana.train,RF.R2.espana.train),
  IS.MAE = c(lin.reg.MAE.espana.train,CART.MAE.espana.train,RF.MAE.espana.train),
  IS.RMSE = c(lin.reg.RMSE.espana.train,CART.RMSE.espana.train,RF.RMSE.espana.train),
  OOS.R2 = c(lin.reg.R2.espana.test,CART.R2.espana.test,RF.R2.espana.test),
  OOS.MAE = c(lin.reg.MAE.espana.test,CART.MAE.espana.test,RF.MAE.espana.test),
  OOS.RMSE = c(lin.reg.RMSE.espana.test,CART.RMSE.espana.test,RF.RMSE.espana.test)
)

print(summary_statistics)

Lasso regression to lower variables needed

library(glmnet)

cv_model <- cv.glmnet(data.matrix(espana.pisa[,2:94]),espana.pisa$score, alpha = 1)

best_lambda <- cv_model$lambda.min
best_lambda
plot(cv_model)

country <- glmnet(data.matrix(espana.pisa[,2:94]),espana.pisa$score, alpha = 1, lambda = best_lambda)
coef(country)

y_predicted <- predict(country, s = best_lambda, newx = data.matrix(espana.test[,2:94]))

#find SST and SSE
sst <- sum((espana.test$score - mean(espana.test$score))^2)
sse <- sum((y_predicted - espana.test$score)^2)

#find R-Squared
rsq <- 1 - sse/sst
rsq

cv_all <- cv.glmnet(data.matrix(clean.data[,1:94]),clean.data$score, alpha = 1)

all.lambda <- cv_all$lambda.min
all.lambda
plot(cv_all)

all.country <- glmnet(data.matrix(train[,1:94]),train$score, alpha = 1, lambda = all.lambda)
coef(all.country)

y_predicted <- predict(all.country, s = all.lambda, newx = data.matrix(test[,1:94]))

#find SST and SSE
sst <- sum((test$score - mean(test$score))^2)
sse <- sum((y_predicted - test$score)^2)

#find R-Squared
rsq <- 1 - sse/sst
rsq

library(iai)

Attaching package: ‘iai’

The following object is masked from ‘package:stats’:

    predict

The following objects are masked from ‘package:base’:

    apply, transform

iai::install_julia()

trying URL 'https://julialang-s3.julialang.org/bin/versions.json'
Content type 'application/json' length 903543 bytes (882 KB)
==================================================
downloaded 882 KB

trying URL 'https://docs.interpretable.ai/versions.json'
Content type 'application/json' length 58743 bytes (57 KB)
==================================================
downloaded 57 KB

trying URL 'https://julialang-s3.julialang.org/bin/mac/x64/1.8/julia-1.8.3-mac64.dmg'
Content type 'application/x-apple-diskimage' length 108113693 bytes (103.1 MB)
==================================================
downloaded 103.1 MB

[1] "Installed Julia to /Users/mauriciodarcourt/Library/Application Support/InterpretableAI/julia/1.8.3/Julia-1.8.app/Contents/Resources/julia"

iai::install_system_image()

In order to continue the installation process, please review the license agreement.

yes

trying URL 'https://docs.interpretable.ai/versions.json'
Content type 'application/json' length 58743 bytes (57 KB)
==================================================
downloaded 57 KB

trying URL 'https://iai-system-images.s3.amazonaws.com/macos/julia1.7.2/v3.1.1/sys-macos-julia1.7.2-iai3.1.1.zip'
Content type 'application/zip' length 91964085 bytes (87.7 MB)
==================================================
downloaded 87.7 MB

Installed IAI system image to /Users/mauriciodarcourt/Library/Application Support/InterpretableAI/sysimage/v3.1.1
┌ Error: No loaded BLAS libraries were built with ILP64 support
└ @ LinearAlgebra.BLAS /Users/julia/buildbot/worker/package_macos64/build/usr/share/julia/stdlib/v1.7/LinearAlgebra/src/blas.jl:154

Quitting.

grid <- iai::grid_search(
    iai::optimal_tree_regressor(
        random_seed = 123,
    ),
    max_depth = 1:5,
)

iai::predict(grid, test[,1:94])
iai::score(grid, train_X, train_y, criterion = "mse")
iai::score(grid, test_X, test_y, criterion = "mse")